package cn.edu.hfut.dmic;

import cn.edu.hfut.dmic.util.FileDownload;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.hutool.http.HttpUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.jsoup.select.Evaluator;

import java.io.*;
import java.util.*;

/**
 * Crawling news from hfut news
 *
 * @author hu
 */
public class AutoNewsCrawler extends BreadthCrawler {
    /**
     * @param crawlPath crawlPath is the path of the directory which maintains
     *                  information of this crawler
     * @param autoParse if autoParse is true,BreadthCrawler will auto extract
     *                  links which match regex rules from pag
     */
    private  String urls="https://www.okfree.men/search.php?action=search&word=%E6%9D%8E%E8%8E%8E&scope=all";
    private  String url="https://www.okfree.men/";
    public AutoNewsCrawler(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
        this.addSeed(urls);
        this.addRegex("-.*\\.(jpg|png|gif).*");
        setThreads(50);
        getConf().setTopN(100);
    }

    @Override
    public void visit(Page page, CrawlDatums next) throws Exception {
        String urlss = page.url();

        Document doc = page.doc();
 /*       String title = doc.select("div.cover").select("img[alt]").attr("alt");
        String path="F:\\" + title +"\\"+title+".txt";
        File file = new File(path);
        file.getParentFile().mkdirs();
        file.createNewFile();*/
        Elements elements = doc.select("div.layout_box").select("table.td_line").select("tr");
        List<String> list = new LinkedList<String>();
        for (Element element : elements) {
            Elements select = element.select("tr.color1").select("td").select("a[href]").eq(0);
            Elements selects = element.select("tr.color4").select("td").select("a[href]").eq(0);
            if (!select.isEmpty()){
                String hreg = select.attr("href");
                String text = select.text();
                list.add(url+hreg+"asmr"+text);
            }
            if (!selects.isEmpty()){
                String hreg2 = select.attr("href");
                String text = select.text();
                list.add(url+hreg2+"asmr"+text);
            }

        }
//        OutputStreamWriter otp = new OutputStreamWriter(new FileOutputStream(path),"UTF-8");
       /* for (Element e : elements) {
            Elements select = e.select("li[data-soundurl64]");
            for (Element element : select) {
                String attr = element.attr("data-soundurl64");
                String bbt = element.attr("title");
                String urls="http://static.missevan.com/"+attr;
                System.out.println(bbt);
                System.out.println(urls);
                String filePath="F:\\"+bbt+".mp3";
                File f = new File(filePath);
                if (!f.exists()) {
                    //调用下载，下载资源
                    FileDownload.downloadFile(urls, "F:\\" + bbt + ".mp3");
                }
                try {
                    otp.write(urls);
                    otp.write("\r\n");
                    otp.flush();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }

            }
        }*/
    }

    public static void main(String[] args) throws Exception {
        Map<String, Object> map = new HashMap<String, Object>();

        map.put("action","login");
        map.put("task","login");
        map.put("ref","https://www.okfree.men/");
        map.put("formhash","0af1aa15");
        map.put("username","canger");
        map.put("password","a4298675");
        String post = (String) HttpUtil.post("https://www.okfree.men/account.php",map);
        System.out.println("登录"+post);
        AutoNewsCrawler crawler = new AutoNewsCrawler("crawl", true);
        /*start crawl with depth of 4*/
        crawler.start(4);
    }

}
